Stock Price Forecasting

Exploring the Time Series Problem of stock market forecasting.

All Datasets available @

BTC, AAPL, MSFT, TSLA, ^IXIC(NASDAQ), ^BVSP(IBOVESPA):
https://finance.yahoo.com/

S&P 500:
https://www.kaggle.com/datasets/andrewmvd/sp-500-stocks?select=sp500_index.csv

Group Members:

200028880 - Wallace Ben Teng Lin Wu

222011561 - Mateus Elias de Macedo

222011525 - Erick Hideki Taira

221029051 - Rodrigo Marques Maranhao

Preprocessing

import pandas as pd
import datetime

def str_to_datetime(s):
    """ Converts a string object to the respective datetime object"""

    year, month, day = [int(i) for i in s.split('-')]
    return datetime.datetime(year=year, month=month, day=day)


price_dict = {
    "Adj Close" : "Price",
    "S&P500" : "Price",
}

def load_df(filename):
    """
    Create a pandas dataframe, filter to leave only the Price column,
    convert date to datetime and make it the index
    """

    df = pd.read_csv(filename)
    df.rename(columns = price_dict, inplace = True)

    # Univariate analysis
    df = df[["Date", "Price"]]

    # Convert date type objects to datetime object
    df["Date"] = df["Date"].apply(str_to_datetime)

    # Turn "Date" Column into dataframe index
    df.index = df.pop("Date")

    return df.dropna()


df = load_df("Datasets/MSFT.csv")

df

	Price
Date
1986-03-13	0.060396
1986-03-14	0.062553
1986-03-17	0.063632
1986-03-18	0.062014
1986-03-19	0.060936
...	...
2023-11-01	346.070007
2023-11-02	348.320007
2023-11-03	352.799988
2023-11-06	356.529999
2023-11-07	360.529999

9491 rows × 1 columns

import matplotlib.pyplot as plt

plt.figure(figsize=(10,6))
plt.plot(df.index, df["Price"])
plt.title("Full Dataset")
plt.show()

# Choose the amount of days to consider from the dataset
days = 5000 # ~13 years

# numbers of days to consider in the input of the model
lookback = 15 #


def df_to_windowed(fullDF, n=lookback, daysSelected=days):
    """
    Create a windowed Dataframe (converting into a supervised problem).
    Therefore, the last {lookback} days prices will be the (input)
    and will generate the next day price (output)
    """

    tmp_df = pd.DataFrame()
    for i in range(n, 0, -1):
        tmp_df[f"Last-{i} Price"] = fullDF["Price"].shift(periods=i)
    tmp_df["Price"] = fullDF["Price"]

    return tmp_df.dropna()[-daysSelected:]


windowed_df = df_to_windowed(df)

windowed_df

	Last-15 Price	Last-14 Price	Last-13 Price	Last-12 Price	Last-11 Price	Last-10 Price	Last-9 Price	Last-8 Price	Last-7 Price	Last-6 Price	Last-5 Price	Last-4 Price	Last-3 Price	Last-2 Price	Last-1 Price	Price
Date
2003-12-29	16.282078	16.445030	16.532766	16.664377	16.676914	16.701979	16.758381	16.958931	16.946400	17.172014	17.146950	17.034134	17.015335	16.946400	17.052935	17.209623
2003-12-30	16.445030	16.532766	16.664377	16.676914	16.701979	16.758381	16.958931	16.946400	17.172014	17.146950	17.034134	17.015335	16.946400	17.052935	17.209623	17.247215
2003-12-31	16.532766	16.664377	16.676914	16.701979	16.758381	16.958931	16.946400	17.172014	17.146950	17.034134	17.015335	16.946400	17.052935	17.209623	17.247215	17.153221
2004-01-02	16.664377	16.676914	16.701979	16.758381	16.958931	16.946400	17.172014	17.146950	17.034134	17.015335	16.946400	17.052935	17.209623	17.247215	17.153221	17.203354
2004-01-05	16.676914	16.701979	16.758381	16.958931	16.946400	17.172014	17.146950	17.034134	17.015335	16.946400	17.052935	17.209623	17.247215	17.153221	17.203354	17.635786
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
2023-11-01	332.420013	331.160004	327.730011	332.640015	332.059998	330.109985	331.320007	326.670013	329.320007	330.529999	340.670013	327.890015	329.809998	337.309998	338.109985	346.070007
2023-11-02	331.160004	327.730011	332.640015	332.059998	330.109985	331.320007	326.670013	329.320007	330.529999	340.670013	327.890015	329.809998	337.309998	338.109985	346.070007	348.320007
2023-11-03	327.730011	332.640015	332.059998	330.109985	331.320007	326.670013	329.320007	330.529999	340.670013	327.890015	329.809998	337.309998	338.109985	346.070007	348.320007	352.799988
2023-11-06	332.640015	332.059998	330.109985	331.320007	326.670013	329.320007	330.529999	340.670013	327.890015	329.809998	337.309998	338.109985	346.070007	348.320007	352.799988	356.529999
2023-11-07	332.059998	330.109985	331.320007	326.670013	329.320007	330.529999	340.670013	327.890015	329.809998	337.309998	338.109985	346.070007	348.320007	352.799988	356.529999	360.529999

5000 rows × 16 columns

windowed_df["Price"].describe()

count    5000.000000
mean       80.652762
std        92.782729
min        11.327569
25%        20.142975
50%        30.797449
75%       102.435174
max       360.529999
Name: Price, dtype: float64

def split_xy(windowedNP):
    """
    Split np.array into X and y
    """

    X = windowedNP[:, :-1]
    y = windowedNP[:, -1]
    return (X, y)

Standardization (Padronização)

Standardization != Normalization

Padronização != Normalização

from IPython import display
display.Image("Images/Standardization.png")

from sklearn.preprocessing import StandardScaler

def scale_data(train, vali, test):
    """ Get Scaled Data """ 
    
    scaler = StandardScaler()
    X_train, y_train = split_xy(scaler.fit_transform(train))
    X_vali, y_vali = split_xy(scaler.transform(vali))
    X_test, y_test = split_xy(scaler.transform(test))
    return scaler, [X_train, X_vali, X_test], [y_train, y_vali, y_test]

def descale_data(train, vali, test, pred, scaler):
    """ Get de-Scaled Data """ 
    X_train, y_train = split_xy(train.to_numpy())
    X_vali, y_vali = split_xy(vali.to_numpy())
    X_test, y_test = split_xy(test.to_numpy())
    X_result, y_result = split_xy(scaler.inverse_transform(pred))
    return [y_train, y_vali, y_test, y_result]

Models

Theory

1 Dimensional Convolution:

1D Max Pooling:

display.Image("Images/MaxPooling.png")

display.Image("Images/Conv+Max.png")

# Max_pooling aceita padding também!

Long Short Term Memory (LSTM):

display.Image("Images/LSTM.png")

display.Image("Images/Forget_Gate.png")

display.Image("Images/Input_Gate.png")

display.Image("Images/Output_Gate.png")

Dropout Layer:

Best Models

That survived my manual natural selection...

from tensorflow.keras.models import Sequential
from tensorflow.keras import layers

# model input: (last {lookback} days prices, 1 feature = "price")
models = []

models.append(
    Sequential([ # CNN+LSTM+Dropout
       layers.Input((lookback, 1)),
       layers.Conv1D(128, kernel_size=3, activation="relu", padding="same"),
       layers.MaxPooling1D(pool_size=2, padding="same"),
       layers.LSTM(128, return_sequences=True),
       layers.Flatten(),
       layers.Dropout(0.3),
       layers.Dense(128),
       layers.Dense(1)
    ]),
)
models[-1].summary()

Model: "sequential"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv1d (Conv1D)             (None, 15, 128)           512       
                                                                 
 max_pooling1d (MaxPooling1  (None, 8, 128)            0         
 D)                                                              
                                                                 
 lstm (LSTM)                 (None, 8, 128)            131584    
                                                                 
 flatten (Flatten)           (None, 1024)              0         
                                                                 
 dropout (Dropout)           (None, 1024)              0         
                                                                 
 dense (Dense)               (None, 128)               131200    
                                                                 
 dense_1 (Dense)             (None, 1)                 129       
                                                                 
=================================================================
Total params: 263425 (1.00 MB)
Trainable params: 263425 (1.00 MB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

models.append(
    Sequential([ # LSTM+Dropout
        layers.Input((lookback, 1)),
        layers.LSTM(128, return_sequences=False),
        layers.Dropout(0.3),
        layers.Dense(128),
        layers.Dense(128),
        layers.Dense(1)
    ]),
)
models[-1].summary()

Model: "sequential_1"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 lstm_1 (LSTM)               (None, 128)               66560     
                                                                 
 dropout_1 (Dropout)         (None, 128)               0         
                                                                 
 dense_2 (Dense)             (None, 128)               16512     
                                                                 
 dense_3 (Dense)             (None, 128)               16512     
                                                                 
 dense_4 (Dense)             (None, 1)                 129       
                                                                 
=================================================================
Total params: 99713 (389.50 KB)
Trainable params: 99713 (389.50 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

models.append(
    Sequential([ # CNN
        layers.Input((lookback, 1)),
        layers.Conv1D(128, kernel_size=3, activation="relu", padding="same"),
        layers.MaxPooling1D(pool_size=2, padding="same"),
        layers.Flatten(),
        layers.Dense(128),
        layers.Dense(1)
    ]),
)
models[-1].summary()

Model: "sequential_2"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 conv1d_1 (Conv1D)           (None, 15, 128)           512       
                                                                 
 max_pooling1d_1 (MaxPoolin  (None, 8, 128)            0         
 g1D)                                                            
                                                                 
 flatten_1 (Flatten)         (None, 1024)              0         
                                                                 
 dense_5 (Dense)             (None, 128)               131200    
                                                                 
 dense_6 (Dense)             (None, 1)                 129       
                                                                 
=================================================================
Total params: 131841 (515.00 KB)
Trainable params: 131841 (515.00 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

models.append(
    Sequential([ # Simple Neural Network
        layers.Input((lookback, 1)),
        layers.Flatten(),
        layers.Dense(128),
        layers.Dense(128),
        layers.Dense(1)
    ]),
)
models[-1].summary()

Model: "sequential_3"
_________________________________________________________________
 Layer (type)                Output Shape              Param #   
=================================================================
 flatten_2 (Flatten)         (None, 15)                0         
                                                                 
 dense_7 (Dense)             (None, 128)               2048      
                                                                 
 dense_8 (Dense)             (None, 128)               16512     
                                                                 
 dense_9 (Dense)             (None, 1)                 129       
                                                                 
=================================================================
Total params: 18689 (73.00 KB)
Trainable params: 18689 (73.00 KB)
Non-trainable params: 0 (0.00 Byte)
_________________________________________________________________

Model Training

Auxilary Functions

# For each year, 60% train, 20% validation, 20% test
def sliding_window(windowed, trainSZ=2100, valiSZ=450, testSZ=450, step=20):
    """
    Sliding Window Generator
    """

    for i in range(0, len(windowed) - trainSZ - testSZ - valiSZ + 1, step):
        train_slice = windowed[i : i+trainSZ]
        vali_slice = windowed[i+trainSZ : i+trainSZ+valiSZ]
        test_slice = windowed[i+trainSZ+valiSZ : i+trainSZ+valiSZ+testSZ]
        yield (train_slice, vali_slice, test_slice)

windows_list = list(sliding_window(windowed_df))
windows_cnt = len(windows_list)

plt.figure(figsize=(10,6))
plt.title(f"Number of Selected Windows: {windows_cnt}")
plt.legend([
    "Training Observations",
    "Validation Observations",
    "Testing Observations",
])
for train, vali, test in windows_list:
    plt.axvline(train.index[0], color="tab:gray")
    plt.plot(train.index, train["Price"], color="tab:blue")
    plt.plot(vali.index, vali["Price"], color="tab:orange")
    plt.plot(test.index, test["Price"], color="tab:green")
    plt.axvline(test.index[-1], color="tab:gray")

plt.show()

plt.figure(figsize=(10,6))
train, vali, test = windows_list[-1]
plt.axvline(train.index[0], color="tab:gray")
plt.plot(train.index, train["Price"], color="tab:blue")
plt.plot(vali.index, vali["Price"], color="tab:orange")
plt.plot(test.index, test["Price"], color="tab:green")
plt.axvline(test.index[-1], color="tab:gray")
plt.title(f"Last Window: # {windows_cnt-1}")
plt.legend([
    "Training Observations",
    "Validation Observations",
    "Testing Observations",
])
plt.show()

import numpy as np
from sklearn.metrics import confusion_matrix

def compute_accuracy_and_cm(y_val, y_test, y_pred):
    """
    Computes the accuracy score and the confusion matrix
    For simplicity, zero price change are considered as positive
    """

    sz = len(y_test)
    y_ref = np.append(y_val[-1], y_test)
    
    y_test_label = np.zeros(sz)
    y_pred_label = np.zeros(sz)

    acc = 0
    for i in range(sz):
        y_test_label[i] = 1 if ((y_test[i] - y_ref[i]) >= 0) else -1
        y_pred_label[i]  = 1 if ((y_pred[i] - y_ref[i]) >= 0) else -1

        if y_test_label[i] == y_pred_label[i]:
            acc += 1

    cm = confusion_matrix(y_true=y_test_label, y_pred=y_pred_label)
    return acc/sz, cm

from matplotlib import patches
patienceSelected = 50

def plot_loss_curve(history, model_idx, i, patience=patienceSelected):
    """
    Plots the loss curve for the model fitting process
    """
    
    logs = history.history
    aux_list = [(val, i) for i, val in enumerate(logs['combine_metric'])]
    best = min(aux_list)
    last = len(logs['combine_metric'])

    plt.figure(figsize=(10,6))
    plt.title(f"Loss Curve for: Model {model_idx}, Window {i}")
    plt.plot(logs["loss"], label="Training Loss")
    plt.plot(logs["val_loss"], label="Validation Loss")
    plt.plot(logs["combine_metric"], label="Combined Loss")
    plt.ylabel("Loss")
    plt.xlabel("Epoch")

    plt.axvline(last-1, color="tab:gray", ymax=0.3, linestyle='--')
    plt.axvline(last-patience-1, color="tab:gray", ymax=0.3, linestyle='--')
    plt.axvline(best[1], color="tab:red", ymax=0.3, linestyle='--')
    
    red_patch = patches.Patch(
        color="tab:red", 
        label=f"best epoch={best[1]}")
    
    gray_patch = patches.Patch(
        color="tab:gray", 
        label=f"Early Stop Limits ({last-patience-1}, {last-1})")

    handles, labels = plt.gca().get_legend_handles_labels()
    handles.extend([red_patch, gray_patch])

    plt.legend(handles=handles, loc="upper right")
    plt.show()

def plot_predictions(dates, ys, metrics, model_idx, i):
    """
    Plots the predicted curve, comparing with observation data
    """
    
    dates_train, dates_vali, dates_test = dates
    y_train, y_vali, y_test, y_result = ys
    rmse, mae, mape, r2, acc = metrics
    
    plt.figure(figsize=(10,6))
    plt.plot(dates_train, y_train)
    plt.plot(dates_vali, y_vali)
    plt.plot(dates_test, y_test)
    plt.plot(dates_test, y_result)
    plt.legend([
        "Training Observations",
        "Validation Observations",
        "Testing Observations",
        "Testing Predictions"
    ])
    plt.title(f"Model {model_idx}, Window {i} \n \
              RMSE={rmse:.3f}, MAE={mae:.3f}, MAPE={mape:.3f}, R2={r2:.3f}" )
    plt.show()

from sklearn.metrics import ConfusionMatrixDisplay

def plot_confusion_matrix(cm, metrics, model_idx, i):
    """
    Plots the confusion matrix for the price change classification
    """

    rmse, mae, mape, r2, acc = metrics

    cm_plt = ConfusionMatrixDisplay(cm, 
                                    display_labels=["Positive", "Negative"])
    cm_plt.plot()
    cm_plt.ax_.set(
        title= f"Model {model_idx}, Window {i}, Accuracy={acc:.3f}",
        xlabel= "Predicted Price Change",
        ylabel= "Actual Price Change"
    )
    plt.show()

Creating important callbacks that will be used in the model training to avoid overfitting and also redundant longer training:

from keras.callbacks import EarlyStopping , Callback, ModelCheckpoint
import h5py 

class CombineCallback(Callback):
    def __init__(self, **kargs):
        super(CombineCallback, self).__init__(**kargs)
    def on_epoch_end(self, epoch, logs={}):
        f = 0.2 # f=vali_factor, 80% training loss, 20% validation loss
        logs['combine_metric'] = f*logs['val_loss']+(1-f)*logs['loss']

combined_cb = CombineCallback()
model_checkpoint = ModelCheckpoint(
    filepath="Models/tmp_best_model.h5", 
    monitor="combine_metric", 
    mode="min", 
    save_best_only=True, 
    save_weights_only=True,
    verbose=False
)
earlyStop = EarlyStopping(monitor="combine_metric", 
                          min_delta=0, 
                          patience=patienceSelected, 
                          mode="min", 
                          verbose=False)

Main Function

from sklearn.metrics import mean_squared_error, mean_absolute_error
from sklearn.metrics import r2_score, mean_absolute_percentage_error
from tensorflow.keras.optimizers import Adam

def cross_validation(model, generator, model_idx, flag_plot=0):
    """
    Performs Cross validation for all models and all sliding windows

    Calculates the cross validation score:
        ("RMSE", "MAE", "MAPE", "R2", "Accuracy");

    Accuracy is computed by classifying if the relative price change 
    for day i was positive or negative
    """

    cv_score = pd.DataFrame(columns=["RMSE", "MAE", "MAPE", "R2", "Acc"])

    for i, (train, vali, test) in enumerate(generator):
        # Get Dates = [dates_train, dates_vali, dates_test]
        dates = [i.index for i in [train, vali, test]]

        # Scale data
        scaler, X_sc, y_sc = scale_data(train, vali, test)
        X_train_sc, X_vali_sc, X_test_sc = X_sc
        y_train_sc, y_vali_sc, y_test_sc = y_sc

        # Fit, save best model and Predict
        model.load_weights("Models/empty_model.h5", 
                           skip_mismatch=True, by_name=True)
        
        model.reset_states()
        history = model.fit(
            X_train_sc, y_train_sc,
            validation_data=(X_vali_sc, y_vali_sc),
            epochs=200, # maximum number of epochs
            batch_size=64, # better for jumping local minimas
            verbose=False,
            callbacks=[combined_cb, earlyStop, model_checkpoint]
        )
        model.load_weights("Models/tmp_best_model.h5",
                           skip_mismatch=True, by_name=True)
        
        preds_sc = model.predict(X_test_sc, verbose=False)

        # Descale data
        stacked_pred = np.hstack((X_test_sc, preds_sc))
        ys =  descale_data(train, vali, test, stacked_pred, scaler)
        [y_train, y_vali, y_test, y_result] = ys

        # Compute Metrics
        rmse = mean_squared_error(y_test, y_result, squared=False)
        mae = mean_absolute_error(y_test, y_result)
        mape = mean_absolute_percentage_error(y_test, y_result)
        r2 = r2_score(y_test, y_result)
        acc, cm = compute_accuracy_and_cm(y_vali, y_test, y_result)

        metrics = [rmse, mae, mape, r2, acc]

        # Plot All Curves and Metrics; Also loss curves
        if flag_plot == 2:
            plot_loss_curve(history, model_idx, i)
            plot_predictions(dates, ys, metrics, model_idx, i)
            plot_confusion_matrix(cm, metrics, model_idx, i)

        # Plot only 5 Curves and Metrics;
        elif (flag_plot == 1 and (i % (windows_cnt//5)) == 0):
            plot_loss_curve(history, model_idx, i)
            plot_predictions(dates, ys, metrics, model_idx, i)
            plot_confusion_matrix(cm, metrics, model_idx, i)

        # Append Result
        cv_score.loc[len(cv_score)] = metrics

    return cv_score

# For each model, perform a cross validation training,
# plot graphs and compute metrics if wanted
cv_scores = []

def run_model(model, i):
    """ 
    Creates the model with the selected architecture and empty weights,
    compile it with mse loss function and Adam optimizer
    and computes the cross validation results for this model
    """

    model.compile(
        loss="mean_squared_error",
        optimizer=Adam(learning_rate=0.0001)
    )   
    model.save_weights("Models/empty_model.h5")
    generator = sliding_window(windowed_df)
    cv_score = cross_validation(model, generator, i, 1)
    cv_scores.append(cv_score)

Backpropagation Method: Adam Optimizer

"We propose Adam, a method for efficient stochastic optimization that only requires first-order gradients with little memory requirement. The method computes individual adaptive learning rates for
different parameters from estimates of first and second moments of the gradients; the name Adam
is derived from adaptive moment estimation. Our method is designed to combine the advantages
of two recently popular methods: AdaGrad (Duchi et al., 2011), which works well with sparse gradients, and RMSProp (Tieleman & Hinton, 2012), which works well in on-line and non-stationary
settings; 

Some of Adam’s advantages are that the magnitudes of parameter updates are invariant to
rescaling of the gradient, its stepsizes are approximately bounded by the stepsize hyperparameter,
it does not require a stationary objective, it works with sparse gradients, and it naturally performs a
form of step size annealing."

~Diederik P. Kingma, Jimmy Lei Ba (Authors)

ADAM: Adaptive moment estimation

Adaptive momentum and adaptive learning rate !!

# CNN + LSTM + Dropout
run_model(models[0], 0)

# LSTM + Dropout
run_model(models[1], 1)

# CNN
run_model(models[2], 2)

# Simple Neural Network
run_model(models[3], 3)

Results

# Output summary (mean, std, min, max)
for i, cv_score in enumerate(cv_scores):
    print(f"Model {i}")
    print(cv_score.describe(), "\n\n")

Model 0
             RMSE         MAE        MAPE          R2         Acc
count  101.000000  101.000000  101.000000  101.000000  101.000000
mean     3.895234    3.075236    0.021306    0.961728    0.524180
std      2.702409    2.180319    0.009673    0.024796    0.039309
min      0.683601    0.497610    0.008261    0.878070    0.428889
25%      1.149078    0.918554    0.015178    0.941309    0.502222
50%      4.456008    3.609054    0.017775    0.965491    0.524444
75%      5.662584    4.461142    0.027819    0.984983    0.557778
max     10.349789    8.664503    0.043119    0.995182    0.588889 


Model 1
             RMSE         MAE        MAPE          R2         Acc
count  101.000000  101.000000  101.000000  101.000000  101.000000
mean     9.628349    7.874803    0.045159    0.833011    0.459978
std      8.790591    7.196723    0.021031    0.137579    0.030172
min      0.704930    0.502852    0.010714    0.339006    0.411111
25%      2.339971    1.770794    0.027120    0.804450    0.433333
50%      7.281628    6.114446    0.044539    0.876047    0.457778
75%     14.199059   11.176512    0.054969    0.922518    0.486667
max     31.357097   26.573715    0.095304    0.984503    0.517778 


Model 2
             RMSE         MAE        MAPE          R2         Acc
count  101.000000  101.000000  101.000000  101.000000  101.000000
mean     2.699445    2.034105    0.015535    0.975658    0.506491
std      1.663646    1.269412    0.005354    0.027713    0.042541
min      0.642425    0.460840    0.008205    0.781972    0.411111
25%      1.351657    1.027763    0.012711    0.969669    0.480000
50%      1.928585    1.610527    0.014282    0.986385    0.506667
75%      4.011799    2.981743    0.016484    0.990132    0.546667
max      5.880076    4.651696    0.042051    0.995010    0.568889 


Model 3
             RMSE         MAE        MAPE          R2         Acc
count  101.000000  101.000000  101.000000  101.000000  101.000000
mean     2.695449    2.016152    0.014529    0.980740    0.497514
std      1.729265    1.309784    0.003360    0.013720    0.052106
min      0.615531    0.406209    0.007916    0.915134    0.402222
25%      0.959838    0.741015    0.012418    0.973966    0.453333
50%      1.946378    1.578985    0.014739    0.985687    0.493333
75%      4.170105    3.055891    0.015928    0.988714    0.540000
max      5.730381    4.524243    0.026676    0.995530    0.584444 

# Section by Parameter
metrics = ["RMSE", "MAE", "MAPE", "R2", "Acc"]

evaluation = {
    "RMSE": {},
    "MAE": {},
    "MAPE": {},
    "R2": {},
    "Acc": {},
}

for model_idx, cv_score in enumerate(cv_scores):
    for param in metrics:
        evaluation[param][f"Model {model_idx}"] = cv_score[param].mean()

def plot_metric(param, logScale=False):
    """
    Plots the evaluation metrics, comparing each model
    """

    if logScale:
        plt.figure(figsize=(10,6))
        plt.title(param + " in Log Scale")
        plt.bar(list(evaluation[param].keys()), 
                list(evaluation[param].values()), 
                color="tab:orange")
        plt.xlabel("Models")
        plt.ylabel("Metrics Value")
        plt.yscale('log')
        plt.show()

    else:
        plt.figure(figsize=(10,6))
        plt.title(param)
        plt.bar(list(evaluation[param].keys()), 
                list(evaluation[param].values()))
        plt.xlabel("Models")
        plt.ylabel("Metrics Value")
        plt.show()

display.Image("Images/RootMeanSquareError.png")

plot_metric("RMSE")

plot_metric("RMSE", 1)

display.Image("Images/MeanAbsoluteError.png")

plot_metric("MAE")

plot_metric("MAE", 1)

display.Image("Images/MeanAbsolutePercentageError.png")

plot_metric("MAPE")

plot_metric("MAPE", 1)

display.Image("Images/R2-DeterminationCoefficient.png")

plot_metric("R2")

plot_metric("R2", 1)

display.Image("Images/Accuracy.png")

plot_metric("Acc")

plot_metric("Acc", 1)

# Output complete results   
for i, cv_score in enumerate(cv_scores):
    print(f"Model {i}")
    print(cv_score)

Model 0
         RMSE       MAE      MAPE        R2       Acc
0    0.787477  0.577116  0.014655  0.950195  0.515556
1    0.961965  0.735219  0.018258  0.937613  0.526667
2    1.025013  0.775381  0.018874  0.934461  0.526667
3    1.065233  0.807187  0.019402  0.928009  0.524444
4    1.103644  0.843336  0.020038  0.923561  0.526667
..        ...       ...       ...       ...       ...
96   5.897175  4.710589  0.017182  0.968836  0.491111
97   5.945330  4.745758  0.017228  0.971457  0.495556
98   5.960590  4.739940  0.017252  0.970987  0.495556
99   5.867653  4.672015  0.016994  0.971648  0.500000
100  5.835192  4.667708  0.016916  0.973805  0.500000

[101 rows x 5 columns]
Model 1
          RMSE        MAE      MAPE        R2       Acc
0     1.335157   1.191497  0.030293  0.856827  0.493333
1     0.990914   0.788837  0.019502  0.933802  0.493333
2     0.862260   0.643464  0.015650  0.953622  0.495556
3     1.123366   0.930515  0.022504  0.919937  0.495556
4     1.392760   1.214147  0.028957  0.878267  0.491111
..         ...        ...       ...       ...       ...
96    8.098891   6.673524  0.023479  0.941221  0.511111
97    8.752363   7.202692  0.025108  0.938142  0.508889
98    9.411969   7.682160  0.027711  0.927661  0.486667
99   12.457886  10.883138  0.037870  0.872195  0.511111
100   8.773446   7.266590  0.025326  0.940783  0.497778

[101 rows x 5 columns]
Model 2
         RMSE       MAE      MAPE        R2       Acc
0    0.788876  0.593824  0.015155  0.950018  0.493333
1    0.668899  0.460840  0.011601  0.969835  0.491111
2    1.074884  0.912324  0.022541  0.927929  0.493333
3    1.853801  1.723600  0.042051  0.781972  0.495556
4    0.699355  0.483527  0.011640  0.969306  0.502222
..        ...       ...       ...       ...       ...
96   5.713737  4.499804  0.016474  0.970744  0.500000
97   5.821043  4.595650  0.016711  0.972638  0.495556
98   5.880076  4.651696  0.016920  0.971766  0.500000
99   5.755412  4.556611  0.016575  0.972722  0.504444
100  5.690243  4.515196  0.016389  0.975090  0.511111

[101 rows x 5 columns]
Model 3
         RMSE       MAE      MAPE        R2       Acc
0    0.615531  0.406209  0.010443  0.969570  0.484444
1    1.066529  0.912504  0.022910  0.923313  0.493333
2    0.734260  0.531329  0.013139  0.966369  0.493333
3    0.909371  0.715635  0.017559  0.947535  0.504444
4    0.680336  0.464767  0.011244  0.970953  0.522222
..        ...       ...       ...       ...       ...
96   5.584017  4.387769  0.016055  0.972058  0.486667
97   5.685540  4.471663  0.016245  0.973897  0.480000
98   5.730381  4.524243  0.016431  0.973185  0.484444
99   5.622617  4.426225  0.016095  0.973966  0.484444
100  5.579112  4.401072  0.015972  0.976054  0.500000

[101 rows x 5 columns]